# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

import random

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from scrapy import signals
from scrapy.core.downloader.handlers.http11 import TunnelError
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.project import get_project_settings
from twisted.internet import defer
from twisted.internet.error import (ConnectError, ConnectionDone,
                                    ConnectionLost, ConnectionRefusedError,
                                    DNSLookupError, TCPTimedOutError,
                                    TimeoutError)
from twisted.web.client import ResponseFailed

from Freepatentsonline.utils.UserAgent import User_Agent_list
from Freepatentsonline.utils.utils import judge_platform

settings = get_project_settings()



class FreepatentsonlineSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


class FreepatentsonlineDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


class RandomUserAgentMiddleware():
    def __init__(self):
        self.headers = {
            'pragma': 'no-cache',
            'cache-control': 'no-cache',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'sec-fetch-site': 'none',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-user': '?1',
            'sec-fetch-dest': 'document',
            'accept-language': 'zh-CN,zh;q=0.9',
        }

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        self.headers["user-agent"] = random.choice(User_Agent_list)
        # if self.headers.get("referer"):
        #     print("refer...")
        #     del self.headers["referer"]
        request.headers.update(self.headers)
        # print("请求的url为:{}".format(request.url))
        # resp=requests.get(request.url,self.headers)
        # print(resp.text)
        # print(f"medi :{request.headers}")
        return None

class ProcessAllExceptionMiddleware(object):
    ALL_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
                      ConnectionRefusedError, ConnectionDone, ConnectError,
                      ConnectionLost, TCPTimedOutError, ResponseFailed,
                      IOError, TunnelError)

    def __init__(self):
        self.error_urls = set()
        self.ERROR_STATUS_SITES =settings.get("ERROR_STATUS_SITES",[])
        pass

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def process_response(self, request, response, spider):
        # 捕获状态码为40x/50x的response
        if response.status >= 400 and request.meta.get("name") in self.ERROR_STATUS_SITES:
            print("逃逸！！！！")
            return response
        elif response.status >= 400:
            # 在日志中打印异常状态码
            print('url:{}的状态码异常: {}'.format(response.url, response.status))
            # 封装一个response，包含请求代理和错误次数返回给spider
            request = self.error_handle(request)
            # time.sleep(10 * 60)
            return request
        elif response.status == 200 and response.url in self.error_urls:
            self.error_urls.remove(response.url)
        # 其他状态码不处理
        return response

    def process_exception(self, request, exception, spider):
        # 捕获几乎所有的异常
        if isinstance(exception, self.ALL_EXCEPTIONS):
            # 在日志中打印异常类型
            print('url:{},捕获异常: {}'.format(request.url, exception))
            # 封装一个response，包含请求代理和错误次数返回给spider
            request = self.error_handle(request)
            return request

            
        # 打印出未捕获到的异常
        print('not contained exception: %s' % exception)

    def error_handle(self, request):

        # request.meta['error_proxy'] = request.meta['proxy']
        error_time = request.meta.get('error_time', 0)
        RETRY_COUNT = settings.get("MY_RETRY_COUNT", 1)
        print(f"程序重拾次数：{RETRY_COUNT}")
        if error_time >= RETRY_COUNT:
            self.error_urls.add(request.url)
            raise IgnoreRequest
        error_time += 1
        request.meta['error_time'] = error_time
        request.meta['error_status'] = 0
        request.dont_filter = True
        return request

    def spider_opened(self, spider):
        # spider.logger.info('Spider opened: %s' % spider.name)
        # self.handle_log_file()
        pass

    def spider_closed(self, spider):

        # print('l1:{}'.format(spider.l1))
        # print('l2:{}'.format(spider.l2))
        # print('l3:{}'.format(spider.l3))
        # print('l4:{}'.format(spider.l4))
        # l1 = list(self.error_set)
        error_urls = list(self.error_urls)
        for i in error_urls:
            print('{}次报错url:{}'.format(settings.get("MY_RETRY_COUNT"), i))




 
  
class CustomProxyMiddleware:  
    
    def __init__(self) -> None:
        self.platform = judge_platform()
        self.proxies = [  
            'http://127.0.0.1:7890',  
             
        ] 
        self.proxy = self.proxies[0]
        
    def process_request(self, request, spider):  
        # 这里假设你有一个代理列表  
         
        # 随机选择一个代理，或者根据你的需要选择  
        
        if "Windows" == self.platform:
            request.meta['proxy'] = self.proxy